#!/usr/bin/env python
# -*- coding: utf-8 -*-

#!/usr/bin/env python
 
# First run tutorial.glade through gtk-builder-convert with this command:
# gtk-builder-convert tutorial.glade tutorial.xml
# Then save this file as tutorial.py and make it executable using this command:
# chmod a+x tutorial.py
# And execute it:
# ./tutorial.py
 
#import pygtk
#pygtk.require("2.0")
import gtk
import gobject
import requests
from ParseData import ParseData, GetUrl, GetOrder, SortUrl
from GetSiteInfo import GetSiteInfo
from DataBase import DataBase

class DiyBox(gobject.GObject):
    __gsignals__ = {
        "get_start": (gobject.SIGNAL_RUN_FIRST,
                  gobject.TYPE_NONE,
                  (gobject.TYPE_STRING,)),
        "get_stop": (gobject.SIGNAL_RUN_FIRST,
                  gobject.TYPE_NONE,
                  ()),
        "get_ok": (gobject.SIGNAL_RUN_FIRST,
                  gobject.TYPE_NONE,
                  ()),
        "get_save": (gobject.SIGNAL_RUN_FIRST,
                  gobject.TYPE_NONE,
                  (gobject.TYPE_STRING,))
        }
    
    def __init__(self, view):
        gobject.GObject.__init__(self)
        self.view = view
        self.view.connect("load_finished",self._delock)

        #get widget from glade xml file
        builder = gtk.Builder()
        builder.add_from_file("diybox.xml")
        builder.connect_signals({ "on_window_destroy" : gtk.main_quit,
                                  "on_combobox_changed" : self._on_combobox_changed,
                                  "on_entry_tag_activate" : self._on_entry_tag_activate,
                                  "on_button_start_clicked" : self._on_button_start_clicked,
                                  "on_button_stop_clicked" : self._on_button_stop_clicked,
                                  "on_button_save_url_clicked" : self._on_button_save_url_clicked,
                                  "on_button_save_analyze_clicked" : self._on_button_save_analyze_clicked,
                                  "on_button_analyze_clicked" : self._on_button_analyze,
                                  "on_entry_tag_button_press_event" : self._on_entry_tag_button_press_event,
                                  "on_radio_url_toggled" : self._on_radio_url_toggled,
                                  "on_radio_order_toggled" : self._on_radio_order_toggled
                                  })
        self.window = builder.get_object("window")
        self.entry = builder.get_object("entry_tag")
        self.combo = builder.get_object("combobox")
        self.but_save = builder.get_object("button_save_url")
        self.but_analyze = builder.get_object("button_analyze")
        self.but_save_analyze = builder.get_object("button_save_analyze")
        self.win_analyze = builder.get_object("scrolledwindow_analyze")
        self.treeview_url = builder.get_object("treeview_url")
        self.treeview_analyze = builder.get_object("treeview_analyze")
        self.liststore_url = builder.get_object("liststore_url")
        self.liststore_analyze = builder.get_object("liststore_analyze")
        
        #default url type 'google'
        self.type = 'google'
        self.entry.set_text(':>Click Start')
        self.tag = ''
        self._init_view()
        self.window.show()
        
    def _on_radio_url_toggled(self, widget):
        if widget.get_active():
            self.combo.show()
            self.entry.show()
            self.type = 'google'
            self.but_save.set_label('SaveUrl')
            self.but_analyze.show()
            self.but_save_analyze.show()
            self.win_analyze.show()
            self._init_view()
        return

    def _on_radio_order_toggled(self, widget):
        if widget.get_active():
            self.type = 'order'
            self.but_save.set_label('SaveOrderList')
            self.but_analyze.hide()
            self.but_save_analyze.hide()
            self.win_analyze.hide()
            self.combo.hide()
            self.entry.hide()
            self._init_view()
        return

    def _on_combobox_changed(self, widget):
        if self.combo.get_active_text() == 'GetGoogleUrl':
            self.type = 'google'
            self.entry.set_text(':>Click Start')
            self.entry.set_editable(False)
        if self.combo.get_active_text() == 'GetBaiduUrl':
            self.type = 'baidu'
            self.entry.set_text(':>Click Start')
            self.entry.set_editable(False)
        if self.combo.get_active_text() == 'DIYTag':
            self.type = 'diy'
            self.entry.set_text(':>Type a tag Then Start')
            self.entry.set_editable(True)       
        
        self._init_view()
        return

    def _on_entry_tag_button_press_event(self, widget, event):
        if self.combo.get_active_text() == 'DIYTag':
            self.entry.set_text('')
        return

    def _on_entry_tag_activate(self, widget):
        self.tag = self.entry.get_text()
        print 'DIY tag: %s' % self.tag
        self._init_view()
        return

    def _on_button_start_clicked(self, widget):
        self.lock = -1
        self.emit('get_start', self.type) 
        return

    def _on_button_stop_clicked(self, widget):
        self.emit('get_stop')
        return

    def _on_button_analyze(self, widget):
        id = 1
        self.liststore_analyze = []
        for row in self.liststore_url:
            site = row[0]
            #print 'Analyze site: %s' % site
            siteinfo = get_site_info(site)
            self.liststore_analyze.append(siteinfo)
            print [id] + siteinfo
            self._display_data(self.treeview_analyze,  self.store_analyze,  self.liststore_analyze)
            id += 1

        print 'Analyze finshed ...'
        return

    def _on_button_save_url_clicked(self, widget):
        if widget.get_label() == 'SaveUrl':
            save_file(self.liststore_url)
        else:
            save_file(self.liststore_order)
        return

    def _on_button_save_analyze_clicked(self, widget):
        save_file(self.liststore_analyze)
        return

    def _init_view(self):
        print 'Type is %s' % self.type
        self.url_dict = {}
        self.lock = 2
        if self.type !='order':
            print 'Init url view ...'
            #self.store_url = gtk.ListStore(int, str, str)
            self.store_analyze = gtk.ListStore(int, str, str, str, str, str, str, str, str, str, str)
            self._display_column(self.treeview_url, ['id', 'url', 'content'])
            self._display_column(self.treeview_analyze,['id', 'site', 'rank', 'PR', 'IPPV', 'BR', 'BDIP', 'domain',
                                                    'created', 'expires', 'regist_name'])
 
            self.do = GetUrl(self.view, self.type) 
            self.nextpage_tag = self.do.nextpage_tag
                    
            if self.type == 'diy':
                self.do = GetUrl(self.view, self.type, self.tag) 

        else:
            print 'Init order view ...'
            self.do = GetOrder(self.view)
            self.nextpage_tag = self.do.nextpage_tag
            self._display_column(self.treeview_url, self.do.column_names)
            self.liststore_order = gtk.ListStore(str, str, str, str, str, str)

        return
            
    def _display_column(self, treeview, columns=[]):
        #view = gtk.TreeView()
        for col in treeview.get_columns():
            treeview.remove_column(col)

        cell = gtk.CellRendererText()
        tvcolumn = [None] * len(columns)
        for n in range(len(columns)):
            tvcolumn[n] = gtk.TreeViewColumn(columns[n], cell, text=n)
            tvcolumn[n].set_sort_column_id(n)
            treeview.append_column(tvcolumn[n])
        return

    def _display_data(self, treeview, storeformat = gtk.ListStore(int, str, str), data = []):
        storeformat.clear()
        id = 1
        if len(data) > 0:
            for row in data:
                storeformat.append([id] + row) 
                id += 1
            treeview.set_model(storeformat)
        return

    def _add_url(self, url_dict_new):
        url = SortUrl().parse_url_dict(url_dict_new)
        for u in url:
            SortUrl().make_url_dict(self.url_dict, [u[0] + uri for uri in u[1]], u[2])

        self.liststore_url = [[u[0] + u[1][0], u[2]] for u in SortUrl().parse_url_dict(self.url_dict)]
        self._display_data(self.treeview_url, data = self.liststore_url)

    def _delock(self, *args):
        #print 'pre lock: %s' % self.lock
        self.lock += 1
        self.emit('get_start', self.type)
        #self._run()
        return
        
    def _get_next_page(self, *args):
        self.url = self.do.get_next_page_url(self.nextpage_tag)
        #print 'Next url: %s' % self.url
        if self.url:
            self.lock -= 1              #Lock until load page ok: 0 start first, -2 lock, -1 start on going
            self.view.open(self.url)
        else:
            self.emit('get_stop')

        return
        
    def _get_page_data(self,  *args):
        if self.lock == -1:
            print 'start to get %s data ...' % self.type
            page_data = self.do.get_page_data()
            print page_data
            if page_data and self.type != 'order':
                self._add_url(page_data)
            else:
                '''
                for order in page_data:
                    self.liststore_order.append(order)
                '''
                self.liststore_order = page_data
                self._display_data(self.treeview_url, gtk.ListStore(int, str, str, str, str, str, str), self.liststore_order)
            self.emit('get_ok')

        return

    def _get_next_page_data(self,  *args):
        if self.lock == -1:
            print 'start get next page ...'
            self._get_next_page()
            self._get_page_data()
        return 

    def _get_stop(self, *args):
        print 'Get data OK!'
        self.lock = 2
        return
        
def save_file(liststore, fp = 'data.txt'):
    for data in liststore:
        if len(data)>1:
            d = ','.join([str(s) for s in data])
        else:
            d = data[0]
        open(fp,'a').write(d + '\n')
    open(fp,'a').write(80*'=' + '\n')
    return
    
def get_site_info(site):
    domain, created, expires, regist_name, rank, PR, IPPV, BR, BDIP = [None]*9

    siteinfo = GetSiteInfo(site).run()
    site_ = siteinfo['site']
    if siteinfo['whois'] != {None}:
        domain = siteinfo['whois']['domain']
        created =  siteinfo['whois']['created']
        expires = siteinfo['whois']['expires']
        regist_name = siteinfo['whois']['registrant']['name']
    if siteinfo['alexa'] != {None}:
        rank = siteinfo['alexa']['rank']
        PR = siteinfo['alexa']['PR']
        IPPV = siteinfo['alexa']['IPPV']
    if siteinfo['baidu_rank']:
        BR = siteinfo['baidu_rank']['BR']
        BDIP = siteinfo['baidu_rank']['BDIP']
    return [site_, rank, PR, IPPV, BR, BDIP, domain, created, expires, regist_name]


def get_url(view, url_type, diy_tag=None):
    return GetUrl(view, url_type, diy_tag).run()
 
if __name__ == "__main__":
  app = DiyBox()
  #app.connect("get_start", ha)
  gtk.main()
